#!/usr/bin/python3
# -*- coding: UTF-8 -*-

import os
import json
from datetime import datetime
import requests
from requests.exceptions import Timeout
import chardet
from urllib.parse import urlparse, unquote, parse_qs
import threading
import time

import data.config as CONFIG

class Spider:
    headers = {
       'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8','Accept-Language': 'en','User-Agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.116 Safari/537.36' 
    }
    
    # 网站请求测试
    def ajax(self,url):
        try:
            response = requests.get(url, headers=self.headers,timeout=(15, 27))
            txt = self.text_encoding(response.content)
            txt = self.text_translation(txt)
            return {"code":200,"data":txt}
        except Timeout:
            return {"code":504,"data":"请求超时"}
        except requests.RequestException as e:
            return {"code":408,"data":e}


    # html转译
    def text_translation(self,txt):
        txt = txt.replace("<","&lt;").replace(">","&gt;")
        return txt

    # 内容解码
    def text_encoding(self,txt):
        try:
            result = chardet.detect(txt)
            encoding = result['encoding']
            print(encoding)
            return txt.decode(encoding)
        except UnicodeDecodeError:
            try:
                return txt.decode('utf-8')
            except UnicodeDecodeError:
                try:
                    return txt.decode('gb18030')
                except UnicodeDecodeError:
                    return txt.decode('ASCII')


    # 连通测试
    def testurl(self,url):
        return self.ajax(url)

    # 自定义列表采集 供给前台
    def getListData(self,data,parsenum):
        folder = CONFIG.path+"/data/data/"+data["config"]
        
        data['folder'] = folder
        self.folder = folder
        thread = threading.Thread(target=self.basethreading, args=(self.getListDataSpider,(data,parsenum),self.getListDataSpiderCallback))
        thread.start()

        return folder
        
    # 线程基类
    def basethreading(self,funname,data,callback):
        thread = threading.Thread(target=funname, args=data)
        thread.start()
        thread.join()
        callback()

    # 爬取列表页
    def getListDataSpider(self,data,parsenum = -1):
        folder = self.folder
        filelog = folder + "/list.log"

        
        print("线程运行中")
        for i in range(1,11):
            print(i)
            self.file(filelog,f"{i}\n","a")
            time.sleep(3)
        
    # 爬取列表页结束调用
    def getListDataSpiderCallback(self):
        print("线程结束")
        
    # 文件操作 w覆盖创建 a追加创建
    def file(self,filename,txt=10100,auth="r"):
        dir_path = os.path.dirname(filename)
        if not os.path.isdir(dir_path):
            os.makedirs(dir_path)
        with open(filename, auth, encoding='utf-8') as file:
            file.write(txt)
        return True
        
    # 临时缓存变量 用于全局
    def runtimedata(self,name,data=10100):
        runtimelog = self.folder + "/runtime.log"
        
    def addconfig(self):    
        now = datetime.now()
        name = now.strftime("%Y-%m-%d_%H-%M-%S")
        dir_path = CONFIG.path+"/data/data/"+name
        if not os.path.isdir(dir_path):
            os.makedirs(dir_path)
        return name
        
